#!/bin/bash

# $Header$
# Author: yang.jin@ebay.com



export PATH=/bin:/usr/bin:/opt/bind/bin
DNS_SERVERS_LIST="/etc/named/IT_DNS_SERVERS"
TESTDOMAIN="vip.its.ebay.com"
PRIMARY="phxitdns01.vip.its.ebay.com"

if [ $# -ne 1 ]; then 
	echo "
		$(basename $0) - Verify SSH/DNS service on remote hosts from a master 

		Usage:
			
		$(basename $0) -a|server[,server...]

			-a	- verify servcies on all servers listed on $DNS_SERVERS_LIST
"
	exit 1	
fi

if [ ! -f $DNS_SERVERS_LIST ]; then
        echo -e "The configuration file \"$DNS_SERVERS_LIST\" does not exist."
        exit 2
fi

. $DNS_SERVERS_LIST


if [ "X$1" == "X-a" ]; then 
	servers=$(echo $DNS_PRIMARY_MASTER $DNS_PSEUDO_MASTERS $DNS_SLAVES)
else
	servers=$(echo $1 | sed -e 's/,/ /g')
fi

if [ "X$(uname -s)" == "XSunOS" ]; then
	GREP="/usr/xpg4/bin/grep"
else
	GREP="/bin/grep"
fi

LOCK="/var/named/run/verify_dns_servers.LOCK"

typeset -i h m lock now INTERVAL=5
h=$(date  -u "+%H" | sed -e 's/^0*//')
m=$(date -u "+%M" | sed -e 's/^0*//')
now=$h*60+$m
if [ -f $LOCK ]; then 
	lock=$(cat $LOCK)
	if [ $lock -ge $now ]; then now=$now+24*60; fi
	if [ $(($now-$lock)) -gt $INTERVAL ]; then
		echo "$(date) Warning: $(basename $0) has been running over $INTERVAL minutes." >&2
		if [ ! $SSH_TTY ]; then 
			echo "$(date) Warning: $(basename $0) has been running over $INTERVAL minutes."  | mailx -s "[unixmon] DNS server verification failed on $(hostname)" itunix@ebay.com
		fi
		exit 3
	else
		echo "$(date) Warning: $(basename $0) has been running. Exiting..." >&2
		exit 0
	fi
fi


local=$(host $(hostname) | awk '/has address/ {print $1}')
localip=$(host $(hostname) | awk '/has address/ {print $4}')
if [ "X${local}" == "X"  -o "X${localip}" == "X" ]; then echo "Error: hostname $(hostname) cannot be resolved." >&2; exit 1; fi

echo -e "$(date) Job $$ starts on ${local}(${localip})"
echo $now > $LOCK

ERRFILE="/tmp/$$.errors"
echo > $ERRFILE

SUCCESS=TRUE
for server in $(echo $servers); do 

	serverip=$(host $server | awk '/has address/ {print $4}')
	serverfqdn=$(host $server | awk '/has address/ {print $1}')
	if [ "X${serverfqdn}" == "X"  -o "X${serverip}" == "X" ]; then 
		echo "Error: ${server} cannot be resolved." >&2
		continue
	fi
	server=$(echo $serverfqdn | cut -d"." -f1)

	echo -e "$(date) Verify services of ${serverfqdn}(${serverip}) -"


	echo -e "$(date)\tverifying SSH at 22/TCP with ssh...\t\c"
	if ssh -q -o ConnectTimeout=30 -o BatchMode=yes -o StrictHostKeyChecking=no $serverfqdn "hostname" 2>/dev/null | $GREP -q "^${server}$"; then
		echo Passed
	else
		SUCCESS=FALSE
		echo Failed
		if [ "X$SSH_TTY" == "X" ]; then 
			echo -e "$(date) PID:$$ Verify services of ${serverfqdn}: SSH failed\n" >> $ERRFILE
		fi
	fi

	echo -e "$(date)\tverifying DNS at 53/TCP with dig...\t\c"
	if dig @${serverfqdn} -t soa $TESTDOMAIN +tcp +short | $GREP -q "^${PRIMARY}"; then
		echo Passed
	else
		SUCCESS=FALSE
		echo Failed
		if [ "X$SSH_TTY" == "X" ]; then 
			echo -e "$(date) PID:$$ Verify services of ${serverfqdn}: dig at 53/TCP failed\n" >> $ERRFILE
		fi
	fi

	echo -e "$(date)\tverifying DNS at 53/UDP with dig...\t\c"
	if dig @${serverfqdn} -t soa $TESTDOMAIN +notcp +short | $GREP -q "^${PRIMARY}"; then
		echo Passed
	else
		SUCCESS=FALSE
		echo Failed
		if [ "X$SSH_TTY" == "X" ]; then 
			echo -e "$(date) PID:$$ Verify services of ${serverfqdn}: dig at 53/UDP failed\n" >> $ERRFILE
		fi
	fi

	echo -e "$(date)\tverifying DNS query to Internet...\t\c"
	if dig @${serverfqdn} -t soa yahoo.com +tcp +short | $GREP -q "^ns1.yahoo.com."; then
		echo Passed
	else
		sleep 5
		if dig @${serverfqdn} -t soa yahoo.com +tcp +short | $GREP -q "^ns1.yahoo.com."; then
			echo Passed
		else
			SUCCESS=FALSE
			echo Failed
			if [ "X$SSH_TTY" == "X" ]; then 
				echo -e "$(date) PID:$$ Verify services of ${serverfqdn}: querying Internet failed\n" >> $ERRFILE
			fi
		fi
	fi
done

if [ $SUCCESS == "FALSE" -a "X$SSH_TTY" == "X" ]; then 
	(cat $ERRFILE; echo -e "\nPlease refer to /ebay/bind-logs/verify_server.log for more details." ) | mailx -s "[unixmon] DNS server verification failed on $(hostname)" itunix@ebay.com
fi

echo -e "$(date) Job $$ completes on ${local}(${localip})"
rm -f $LOCK $ERRFILE
exit 0
